package com.wyl.spider.processor;

import java.util.Arrays;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

public class SimpleProcessor implements PageProcessor{

	private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setTimeOut(10000).setCharset("gbk")
			.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
			.addHeader("Accept-Encoding", "gzip, deflate, sdch")
			.addHeader("Accept-Language", "zh-CN,zh;q=0.8")
			.addHeader("Cache-Control","max-age=0")
			.addHeader("Connection", "keep-alive")
			.addHeader("Upgrade-Insecure-Requests", "1")
			.addHeader("User-Agent",
					"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36");
	
	@Override
	public void process(Page page) {
		page.putField("channel", page.getUrl());
		page.putField("data", page.getJson().toString().substring(14,page.getJson().toString().length()-1));
	}

	@Override
	public Site getSite() {
		return site;
	}
	
}
